import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm;
import org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithmDescriptor;
import org.carrot2.clustering.lingo.LingoClusteringAlgorithm;
import org.carrot2.core.Cluster;
import org.carrot2.core.Controller;
import org.carrot2.core.ControllerFactory;
import org.carrot2.core.IDocumentSource;
import org.carrot2.core.LanguageCode;
import org.carrot2.core.ProcessingResult;
import org.carrot2.core.attribute.CommonAttributesDescriptor;
import org.carrot2.text.linguistic.DefaultLexicalDataFactoryDescriptor;
import org.carrot2.text.linguistic.LexicalDataLoaderDescriptor;
import org.carrot2.text.preprocessing.CaseNormalizerDescriptor;
import org.carrot2.util.resource.DirLocator;
import org.carrot2.util.resource.ResourceLookup;
import org.w3c.dom.Attr;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

import com.google.common.collect.Maps;

//import examples.org.carrot2.examples.ConsoleFormatter;

public class Clustering {

	public static void main( String[] args ){

		boolean useLingoAlgorithm = false;
		boolean useKMeansAlgorithm = true;
		boolean useResources = true;
		String inputURL = "http://localhost:8080/exist/rest/db/projdb/input-data-for-clustering.xml";
		String input = "inputDataForClustering.xml";
		String output = "clusterdata.xml";
		String speaker = "";

		ArrayList<org.carrot2.core.Document> documents = new ArrayList< org.carrot2.core.Document>();
		ArrayList< SpeakerData > allSpeakerData = new ArrayList< SpeakerData >();
		HashMap<String, org.carrot2.core.Document> allDocumentsById = new HashMap<String, org.carrot2.core.Document>();

		readDocuments(allSpeakerData, allDocumentsById, input, inputURL);
		addStopwords(allSpeakerData);

		Controller controller = null;
		if ( useResources )
			controller = loadResources();
		else
			controller = ControllerFactory.createSimple();

		try {
			DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
			DocumentBuilder docBuilder = docFactory.newDocumentBuilder();

			Document xmlDocumentResults = docBuilder.newDocument();
			Element rootElement = xmlDocumentResults.createElement("speakers");
			xmlDocumentResults.appendChild(rootElement);

			speaker = "iedereen";
			Iterator< org.carrot2.core.Document > it = allDocumentsById.values().iterator();
			
			while ( it.hasNext() ) {
				    documents.add(  it.next() );
			}			

			Map<String, Object> attributes = new HashMap<String, Object>();
			attributes.put( CommonAttributesDescriptor.Keys.DOCUMENTS, documents );
			
			attributes.put( BisectingKMeansClusteringAlgorithmDescriptor.Keys.CLUSTER_COUNT, 500 ); // max nr cluster.. bovengrens dus
			attributes.put( BisectingKMeansClusteringAlgorithmDescriptor.Keys.CLUSTERS, 50 );

			attributes.put("BisectingKMeansClusteringAlgorithm.labelCount", 10); //[1:10]
			attributes.put("BisectingKMeansClusteringAlgorithm.maxIterations", 1000);
			attributes.put("BisectingKMeansClusteringAlgorithm.partitionCount", 5); //[2:10]
			attributes.put("BisectingKMeansClusteringAlgorithm.useDimensionalityReduction", false );

			attributes.put("TermDocumentMatrixBuilder.maximumMatrixSize", 150000000);
			attributes.put( CaseNormalizerDescriptor.Keys.DF_THRESHOLD, 7); // 7, 9 & 10 leveren de meeste clusters op..
			
			if ( useLingoAlgorithm ) {
				ProcessingResult byTopicClusters = controller.process(attributes, LingoClusteringAlgorithm.class);
				List<Cluster> clustersByTopic = byTopicClusters.getClusters();
				//ConsoleFormatter.displayClusters(clustersByTopic,0);						
				addClusterResultsToXml(speaker, "Lingo Algorithm", xmlDocumentResults, rootElement, clustersByTopic);						
			}

			if ( useKMeansAlgorithm ) {
				if ( documents.size() > 1 ) {
					ProcessingResult byDomainClusters = controller.process(attributes, BisectingKMeansClusteringAlgorithm.class);
					List<Cluster> clustersByDomain = byDomainClusters.getClusters();
					//ConsoleFormatter.displayClusters(clustersByDomain,0);
					addClusterResultsToXml(speaker, "K-means Algorithm", xmlDocumentResults, rootElement, clustersByDomain );
				} else {
					List<Cluster> clustersByDomain = new ArrayList<Cluster>();
					addClusterResultsToXml(speaker, "K-means Algorithm", xmlDocumentResults, rootElement, clustersByDomain );						
				}
			}

			TransformerFactory transformerFactory = TransformerFactory.newInstance();
			Transformer transformer = transformerFactory.newTransformer();
			DOMSource source = new DOMSource( xmlDocumentResults );
			StreamResult result = new StreamResult(new File( output ));
			transformer.transform(source, result);

		} catch (ParserConfigurationException pce) {
			pce.printStackTrace();
		} catch (TransformerConfigurationException e) {
			e.printStackTrace();
		} catch (TransformerException e) {
			e.printStackTrace();
		}
		
	    File f = new File( input );
	    f.delete();
	}

	/*Writes the output to an xml file*/
	private static void addClusterResultsToXml(String speaker, String algorithm, Document xmlDocumentResults, Element rootElement, List<Cluster> clusters ) {
		Element speakerNode = xmlDocumentResults.createElement("speaker");

		Element name = xmlDocumentResults.createElement("name");
		name.appendChild( xmlDocumentResults.createTextNode( speaker ) );

		Element subjects = xmlDocumentResults.createElement("subjects");

		Attr attrAlgorithm = xmlDocumentResults.createAttribute("algorithm");
		attrAlgorithm.setValue( algorithm );
		subjects.setAttributeNode( attrAlgorithm );

		for ( Cluster cluster : clusters ){
			if ( !cluster.getLabel().equals("Other Topics") && cluster.getLabel().split(", ").length <= 10 ) {
				Element subject = xmlDocumentResults.createElement("subject");
				
				Element label = xmlDocumentResults.createElement("label");
				label.appendChild( xmlDocumentResults.createTextNode( cluster.getLabel() ) );
				subject.appendChild(label);
				
				Element documents = xmlDocumentResults.createElement("documents");
				
				for ( org.carrot2.core.Document d : cluster.getDocuments() ){
					Element document = xmlDocumentResults.createElement("document");
					document.appendChild( xmlDocumentResults.createTextNode( d.getContentUrl() ) );
					documents.appendChild(document);				
				}
				
				subject.appendChild(documents);
				
				subjects.appendChild(subject);
			}
		}

		speakerNode.appendChild(name);
		speakerNode.appendChild(subjects);
		rootElement.appendChild( speakerNode );
	}

	/*Loads additional stopwords specified in the file "carrot/resources/defaultStopwords.nl"*/
	private static void addStopwords(ArrayList<SpeakerData> allSpeakerData) {
		try{
			FileWriter fstreamout = new FileWriter("carrot/resources/stopwords.nl");
			BufferedWriter out = new BufferedWriter(fstreamout);

			FileInputStream fstreamin = new FileInputStream("carrot/resources/defaultStopwords.nl");
			DataInputStream in = new DataInputStream(fstreamin);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			String currentLine;

			while ( ( currentLine = br.readLine() ) != null ) {
				out.write( currentLine + "\n" );
			}

			in.close();

			for (int i = 0; i < allSpeakerData.size(); ++ i ){
				String[] splitedName = allSpeakerData.get(i).getName().split(" ");

				for ( String namePart : splitedName )
					out.write( namePart + "\n" );				
			}

			out.close();
		}catch (Exception e){//Catch exception if any
			System.err.println("Error: " + e.getMessage());
		}
	}

	@SuppressWarnings("unchecked")
	private static Controller loadResources() {
		Controller controller = ControllerFactory.createCachingPooling(IDocumentSource.class);

		File resourcesDir = new File("carrot/resources");
		ResourceLookup resourceLookup = new ResourceLookup(new DirLocator(resourcesDir));

		Map<String, Object> attributesController = Maps.newHashMap();
		DefaultLexicalDataFactoryDescriptor.attributeBuilder( attributesController ).mergeResources(true);
		LexicalDataLoaderDescriptor.attributeBuilder( attributesController ).resourceLookup(resourceLookup);

		controller.init( attributesController );
		return controller;
	}

	/*Reads the input files from the database and stored them in the "allSpeakerData" array*/
	private static void readDocuments( ArrayList< SpeakerData > allSpeakerData, HashMap<String, org.carrot2.core.Document> allDocumentsById, String xmlFile, String URL) {
		try {

			FileWriter fstream = new FileWriter( xmlFile );
			BufferedWriter out = new BufferedWriter(fstream);

			URL urlFile = new URL( URL );
	        BufferedReader in = new BufferedReader(
	        new InputStreamReader(urlFile.openStream()));

	        String inputLine;
	        while ((inputLine = in.readLine()) != null)
	            out.write(inputLine);
	        in.close();			
			
			out.close();
	        
			
			File file = new File( xmlFile );
			if(file.exists()){
				// Create a factory
				DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();

				// Use the factory to create a builder
				DocumentBuilder builder = factory.newDocumentBuilder();
				Document doc = builder.parse( xmlFile );
				Element tree = doc.getDocumentElement();

				NodeList topics = tree.getElementsByTagName( "t:topic" );
				for (int i = 0; i < topics.getLength(); ++i ) {
					Element topic = (Element) topics.item(i);				
					allDocumentsById.put( topic.getElementsByTagName("t:id").item(0).getTextContent() , new org.carrot2.core.Document( topic.getElementsByTagName("t:title").item(0).getTextContent(), topic.getElementsByTagName("t:content").item(0).getTextContent(), topic.getElementsByTagName("t:id").item(0).getTextContent(), LanguageCode.DUTCH ) );
				}

				NodeList speakers = tree.getElementsByTagName( "s:speaker" );
				for (int i = 0; i < speakers.getLength(); ++i ) {
					Element speaker = (Element) speakers.item(i);				
					ArrayList< org.carrot2.core.Document > speakerDocuments = new ArrayList<org.carrot2.core.Document>();

					NodeList ids = speaker.getElementsByTagName("s:id");

					for ( int j = 0; j < ids.getLength(); ++j ) {
						speakerDocuments.add( allDocumentsById.get( ids.item(j).getTextContent() ) );
					}

					allSpeakerData.add( new SpeakerData( speaker.getElementsByTagName("s:name").item(0).getTextContent(), speakerDocuments ) );					
				}
			}
			else{
				System.out.print("File not found!");
			}
		}
		catch (Exception e) {
			System.exit(1);
		}
	}

}
